package com.spider.service;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.spider.dao.HBaseDAOImp;
import com.spider.dao.HBaseDao;
import com.spider.model.Question;
import com.spider.utils.ProcessUtils;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class QuesPageProcess implements PageProcessor{

	private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);
	private ProcessUtils processUtils = new ProcessUtils(); 
	private Matcher match;
	private Pattern pattern;
	private Question question;
	private String ques_id;
	private int flagSize = 0;
	private List<Question> quesList = new ArrayList();
	private HBaseDao dao = new HBaseDAOImp();
	private int index = 0;
	
	
	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}
	
	
	@Override
	public void process(Page page) {
		// TODO Auto-generated method stub
//		this.proContent(page);
		page.putField("html", page.getHtml());
		page.putField("url", page.getUrl());
		String html = page.getResultItems().get("html").toString();
		String urled = page.getResultItems().get("url").toString();
		System.err.println(urled);
		
		List<String> urls = page.getHtml()
				.xpath("//div[@class='laypage_main laypageskin_default']")
				.links().regex(".*hdSearch=.*").all();
		
		List<String> urlz = page.getHtml()
				.xpath("//dl[@class='z-open']")
				.links().regex(".*kid=.*").all();	
		
		for (int i = 0; i < urlz.size(); i++) {
			urls.add(urlz.get(i));
		}
	
		dao.insert("html0714", "contentB"+(index++)+"", "base", "content", html);
		dao.insert("html0714", "htmlurled"+"--"+urled, "base", "url", urled);
		dao.insert("url", "urled"+index+"--"+urled, "url", "urled", urled);
		for(int i = 0;i < urls.size();i++) {
			dao.insert("url", "url"+index+"--"+urled, "urled", "url", urls.get(i));
			index++;
		}
		
		page.addTargetRequests(urls);
	}
	
	public void proContent(Page page){
		page.putField("content", page.getHtml().xpath("//div[@class='u-bodyHtml'")
				.regex("/span&gt;.*").replace("/span&gt;", "")
				.replace("&lt;u&gt;&nbsp;&nbsp;&nbsp; &lt;/u&gt;", "____")
				.replace("&lt;sub&gt;&lt;/sub&gt;", "")
				.replace("&lt;sup&gt;&lt;/sup&gt;", "")
				.replace("&lt;i&gt;","").replace("&lt;/i&gt;","")
				.replace("</div>", "").all());
		
		this.proDiff(page);
		this.proId(page);
		
		String content = processUtils.processStr(page.getResultItems().get("content").toString());
		String[] contents = content.split("\n");
		//��ɴ���
		for (String cont : contents) {
//			question = new Question();
//			flagSize ++;
			String proceUrl = this.processUrl(cont);
			String proce = processUtils.processSub(processUtils.processSup(proceUrl));
			System.err.println(proce);
//			question.setContent(proce);
//			quesList.add(question);
		}
		
		this.proUrl(page);
	}
	
	public void proDiff(Page page){
		page.putField("diff",page.getHtml().xpath("//div[@class='f-fr']/b"));
		String diff = processUtils.processStr(page.getResultItems().get("diff").toString());
		System.out.println("diff:"+diff);
	}
	
	public void proId(Page page){
		page.putField("quesId", page.getHtml().xpath("//li[@class=question]"));
		String rowKey = processUtils.processStr(page.getResultItems().get("quesId").toString());
		System.out.println("rowKey:"+rowKey);
	}
	
	public void proUrl(Page page){
		List<String> urls = page.getHtml()
				.xpath("//div[@class='laypage_main laypageskin_default']")
				.links().regex(".*hdSearch=.*").all();
		page.addTargetRequests(urls);
	}
	
	//�滻imgUrlΪ<img>,���־û�imgUrl
	public String processUrl(String cont){
		String imgUrl = "";
		if(cont.contains("&lt;img")){
			pattern = pattern.compile("http.+png");
			match = pattern.matcher(cont);
			if(match.find()){
				imgUrl = match.group();
//				question.setImgUrl(imgUrl);
			}
			String s = cont.replaceAll("&lt;img.+src.+&gt;", "<img>");
			return s;
		}
		return cont;
	}
	
	public static void main(String[] args) {
		//over
		for(int bid = 800148;bid < 800167;bid++) {
			Spider.create(new QuesPageProcess())
			.addUrl("http://www.tiku.com/testPaper.html?sct=0&cn=%E6%95%B0%E5%AD%A6&st=2&cid=500004&bid="+bid+"&vid=800022&uid=800696&cptid=702233&kid=700124&qtid=600050&difficulty=&sort=0")
			.thread(5)
			.run();
		}
		//overing
//		for(int bid = 800029;bid < 800069;bid++) {
//			Spider.create(new QuesPageProcess())
//			.addUrl("http://www.tiku.com/testPaper.html?sct=0&cn=%E6%95%B0%E5%AD%A6&st=2&cid=500004&bid="+bid+"&vid=800006&uid=800078&cptid=700243&kid=700124&qtid=600050&difficulty=&sort=0")
//			.thread(5)
//			.run();
//		}
		
//		for(int bid = 800803;bid < 800811;bid++) {
//			Spider.create(new QuesPageProcess())
//			
//			.addUrl("http://www.tiku.com/testPaper.html?sct=0&cn=%E6%95%B0%E5%AD%A6&st=2&cid=500004&bid="+bid+"&vid=800101&uid=803748&cptid=711139&kid=700124&qtid=600050&difficulty=&sort=0")
//			.thread(5)
//			.run();
//		}
//		for(int bid = 800655;bid < 80000167;bid++) {
//			Spider.create(new QuesPageProcess())
//			
//			.addUrl("http://www.tiku.com/testPaper.html?sct=0&cn=%E6%95%B0%E5%AD%A6&st=2&cid=500004&bid="+bid+"&vid=800022&uid=800696&cptid=702233&kid=700124&qtid=600050&difficulty=&sort=0")
//			.thread(5)
//			.run();
//		}
//		
//		for(int bid = 800746;bid < 800755;bid++) {

	}
}
